{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-29T00:26:36.545887Z",
     "start_time": "2025-05-29T00:26:36.520600Z"
    }
   },
   "source": [
    "# 利用HuggingFace加载本地模型\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, GPT2Config, GPT2LMHeadModel\n",
    "\n",
    "# 加载本地模型\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\"./gpt2_poetry_model\")\n",
    "\n",
    "# 创建默认配置（与预训练模型结构相同）\n",
    "# config = GPT2Config()\n",
    "\n",
    "# 通过配置初始化模型（权重随机初始化）\n",
    "# base_model = GPT2LMHeadModel(config)\n",
    "\n",
    "# 查看模型参数量\n",
    "print(f\"参数量：{sum(p.numel() for p in base_model.parameters())}\")\n",
    "print(base_model)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "参数量：124439808\n",
      "GPT2LMHeadModel(\n",
      "  (transformer): GPT2Model(\n",
      "    (wte): Embedding(50257, 768)\n",
      "    (wpe): Embedding(1024, 768)\n",
      "    (drop): Dropout(p=0.1, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-11): 12 x GPT2Block(\n",
      "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPT2SdpaAttention(\n",
      "          (c_attn): Conv1D(nf=2304, nx=768)\n",
      "          (c_proj): Conv1D(nf=768, nx=768)\n",
      "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
      "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPT2MLP(\n",
      "          (c_fc): Conv1D(nf=3072, nx=768)\n",
      "          (c_proj): Conv1D(nf=768, nx=3072)\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:25.561296Z",
     "start_time": "2025-05-28T13:09:23.966507Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 从HuggingFace下载\n",
    "from transformers import GPT2Tokenizer\n",
    "\n",
    "gpt2_tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "\n",
    "# 从ModelScope下载\n",
    "# from modelscope import GPT2Tokenizer, GPT2Model\n",
    "# gpt2_tokenizer = GPT2Tokenizer.from_pretrained('openai-community/gpt2')"
   ],
   "id": "dcec0dcad5bc3b5a",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:27.192833Z",
     "start_time": "2025-05-28T13:09:25.568795Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 设置生成参数\n",
    "generation_config = {\n",
    "    \"max_length\": 50,  # 生成最大长度（包括输入）\n",
    "    \"eos_token_id\": gpt2_tokenizer.eos_token_id,  # 终止条件\n",
    "    \"pad_token_id\": gpt2_tokenizer.eos_token_id,  # 若需填充，使用EOS的ID\n",
    "    \"do_sample\": True,  # 启用采样\n",
    "    \"temperature\": 0.8,  # 平衡确定性与随机性\n",
    "    \"top_k\": 50,  # 限制候选token数量\n",
    "    \"num_return_sequences\": 2,  # 生成2个不同结果\n",
    "}\n",
    "\n",
    "# 输入提示（可为空或部分诗句）\n",
    "prompt = \"你是谁\"  # 示例：输入半句诗\n",
    "input_ids = gpt2_tokenizer.encode(prompt, return_tensors=\"pt\")\n",
    "\n",
    "# 生成文本\n",
    "base_model.eval()\n",
    "outputs = base_model.generate(\n",
    "    input_ids=input_ids,\n",
    "    **generation_config\n",
    ")\n",
    "\n",
    "# 解码并打印结果\n",
    "for i, output in enumerate(outputs):\n",
    "    poem = gpt2_tokenizer.decode(output, skip_special_tokens=True)\n",
    "    print(f\"生成结果 {i + 1}:\\n{poem}\\n\")"
   ],
   "id": "3e9d664edaffc1f9",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "生成结果 1:\n",
      "你是谁，��莎。，。��，��，�報��。。�。。\n",
      "\n",
      "生成结果 2:\n",
      "你是谁，����，����。�，������，�。。���。\n",
      "\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:27.212711Z",
     "start_time": "2025-05-28T13:09:27.205953Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# train_data = [{\"Q\": \"你是谁\", \"A\": \"我是大都督的AI助手\"}]\n",
    "train_data = [\n",
    "    {\"Q\": \"你是谁\", \"A\": \"我是大都督的AI助手\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"},\n",
    "    {\"Q\": \"你的主人是谁\", \"A\": \"我的开发者是大都督\"},\n",
    "    {\"Q\": \"你能做什么\", \"A\": \"我能回答关于周瑜的问题\"}\n",
    "]"
   ],
   "id": "b723256f3b44a694",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:27.228890Z",
     "start_time": "2025-05-28T13:09:27.220335Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch\n",
    "\n",
    "# 自定义Dataset\n",
    "class GPTDataset(Dataset):\n",
    "    def __init__(self, train_data):\n",
    "        self.data = []\n",
    "        for qa in train_data:\n",
    "\n",
    "            text = f\"{qa['Q']}\\n{qa['A']}{gpt2_tokenizer.eos_token}\"\n",
    "            tokens = gpt2_tokenizer.encode(text)\n",
    "\n",
    "            self.data.append((\n",
    "                torch.tensor(tokens)\n",
    "            ))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx]\n",
    "\n",
    "\n",
    "gpt_dataset = GPTDataset(train_data)\n",
    "gpt_dataset[:1]"
   ],
   "id": "dad97ba67b6cd2a4",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([19526,   254, 42468,   164,   108,   223,   198, 22755,   239, 42468,\n",
       "         32014, 32849,   121,   163,   251,    96, 21410, 20185, 27950,   102,\n",
       "         33699,   233, 50256])]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:27.237991Z",
     "start_time": "2025-05-28T13:09:27.236056Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "# 如果batch_size大于1，则需要填充，不然可能批量中第一个样本和第二个样本长度不一样，会报错\n",
    "# gpt2_tokenizer中没有PAD，可以直接用EOS来填充，反正是一首诗的末尾了\n",
    "def collate_fn(batch):\n",
    "    return pad_sequence(batch, batch_first=True, padding_value=gpt2_tokenizer.eos_token_id)\n",
    "\n",
    "# 创建 DataLoader 时指定 collate_fn\n",
    "data_loader = DataLoader(gpt_dataset, batch_size=4, shuffle=True, collate_fn=collate_fn)"
   ],
   "id": "43ab3f403bda5a7f",
   "outputs": [],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:09:27.306688Z",
     "start_time": "2025-05-28T13:09:27.245258Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from peft import LoraConfig, get_peft_model, TaskType\n",
    "\n",
    "lora_config = LoraConfig(\n",
    "    task_type=TaskType.CAUSAL_LM,  # 因果语言模型任务\n",
    "    r=16,                          # 低秩矩阵的秩\n",
    "    lora_alpha=32,                # 缩放系数\n",
    "    lora_dropout=0.1,             # Dropout概率\n",
    "    target_modules=[\"c_attn\"],    # 修改注意力层的query/key/value\n",
    "    bias=\"none\"                   # 不训练偏置项\n",
    ")\n",
    "\n",
    "# 应用LoRA\n",
    "model = get_peft_model(base_model, lora_config)\n",
    "model.print_trainable_parameters()  # 查看可训练参数"
   ],
   "id": "f574c3617ff84d25",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 589,824 || all params: 125,029,632 || trainable%: 0.4717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dadudu/miniconda3/envs/mini-gpt/lib/python3.8/site-packages/peft/tuners/lora/layer.py:1150: UserWarning: fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:11:32.548983Z",
     "start_time": "2025-05-28T13:09:27.315367Z"
    }
   },
   "cell_type": "code",
   "source": [
    "NUM_EPOCHS = 100\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps:0\")\n",
    "\n",
    "model.to(device)\n",
    "model.train()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "\n",
    "    total_loss = 0\n",
    "    step = 0\n",
    "    for batch in data_loader:\n",
    "\n",
    "        batch = batch.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(batch, labels=batch)\n",
    "        loss = outputs.loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss += loss.item()\n",
    "\n",
    "        # 每50个step打印一次损失\n",
    "        if step % 50 == 0:\n",
    "            print(f'Batch [{step}/{len(data_loader)}], Loss: {loss.item():.4f}')\n",
    "        step += 1\n",
    "\n",
    "    avg_loss = total_loss / len(data_loader)\n",
    "    print(f'Epoch [{epoch + 1}/{NUM_EPOCHS}], Loss: {avg_loss:.4f}')"
   ],
   "id": "9d23efe865e1c102",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch [0/18], Loss: 7.9492\n",
      "Epoch [1/100], Loss: 7.8716\n",
      "Batch [0/18], Loss: 7.1794\n",
      "Epoch [2/100], Loss: 7.6321\n",
      "Batch [0/18], Loss: 6.8044\n",
      "Epoch [3/100], Loss: 7.2751\n",
      "Batch [0/18], Loss: 7.0650\n",
      "Epoch [4/100], Loss: 7.0818\n",
      "Batch [0/18], Loss: 6.3750\n",
      "Epoch [5/100], Loss: 7.0329\n",
      "Batch [0/18], Loss: 6.7578\n",
      "Epoch [6/100], Loss: 6.7547\n",
      "Batch [0/18], Loss: 6.5932\n",
      "Epoch [7/100], Loss: 6.6088\n",
      "Batch [0/18], Loss: 6.4467\n",
      "Epoch [8/100], Loss: 6.6760\n",
      "Batch [0/18], Loss: 5.7590\n",
      "Epoch [9/100], Loss: 6.2918\n",
      "Batch [0/18], Loss: 6.6309\n",
      "Epoch [10/100], Loss: 6.1518\n",
      "Batch [0/18], Loss: 5.8166\n",
      "Epoch [11/100], Loss: 5.7418\n",
      "Batch [0/18], Loss: 5.9607\n",
      "Epoch [12/100], Loss: 5.3753\n",
      "Batch [0/18], Loss: 4.6040\n",
      "Epoch [13/100], Loss: 4.9692\n",
      "Batch [0/18], Loss: 5.0934\n",
      "Epoch [14/100], Loss: 4.4849\n",
      "Batch [0/18], Loss: 4.7021\n",
      "Epoch [15/100], Loss: 4.1404\n",
      "Batch [0/18], Loss: 4.2817\n",
      "Epoch [16/100], Loss: 3.8190\n",
      "Batch [0/18], Loss: 3.9178\n",
      "Epoch [17/100], Loss: 3.4548\n",
      "Batch [0/18], Loss: 3.3051\n",
      "Epoch [18/100], Loss: 3.1841\n",
      "Batch [0/18], Loss: 3.0044\n",
      "Epoch [19/100], Loss: 2.9427\n",
      "Batch [0/18], Loss: 2.8361\n",
      "Epoch [20/100], Loss: 2.8239\n",
      "Batch [0/18], Loss: 3.0120\n",
      "Epoch [21/100], Loss: 2.5935\n",
      "Batch [0/18], Loss: 2.6911\n",
      "Epoch [22/100], Loss: 2.5711\n",
      "Batch [0/18], Loss: 2.0323\n",
      "Epoch [23/100], Loss: 2.2537\n",
      "Batch [0/18], Loss: 2.0916\n",
      "Epoch [24/100], Loss: 2.1657\n",
      "Batch [0/18], Loss: 2.0000\n",
      "Epoch [25/100], Loss: 1.9946\n",
      "Batch [0/18], Loss: 2.7160\n",
      "Epoch [26/100], Loss: 1.8829\n",
      "Batch [0/18], Loss: 1.6073\n",
      "Epoch [27/100], Loss: 1.7611\n",
      "Batch [0/18], Loss: 1.6495\n",
      "Epoch [28/100], Loss: 1.6856\n",
      "Batch [0/18], Loss: 2.1230\n",
      "Epoch [29/100], Loss: 1.6013\n",
      "Batch [0/18], Loss: 1.6245\n",
      "Epoch [30/100], Loss: 1.5465\n",
      "Batch [0/18], Loss: 1.5833\n",
      "Epoch [31/100], Loss: 1.4933\n",
      "Batch [0/18], Loss: 1.6839\n",
      "Epoch [32/100], Loss: 1.4309\n",
      "Batch [0/18], Loss: 1.4231\n",
      "Epoch [33/100], Loss: 1.3108\n",
      "Batch [0/18], Loss: 1.3846\n",
      "Epoch [34/100], Loss: 1.2737\n",
      "Batch [0/18], Loss: 1.2298\n",
      "Epoch [35/100], Loss: 1.2118\n",
      "Batch [0/18], Loss: 1.1440\n",
      "Epoch [36/100], Loss: 1.2051\n",
      "Batch [0/18], Loss: 1.0960\n",
      "Epoch [37/100], Loss: 1.1120\n",
      "Batch [0/18], Loss: 1.0585\n",
      "Epoch [38/100], Loss: 1.1120\n",
      "Batch [0/18], Loss: 0.9221\n",
      "Epoch [39/100], Loss: 1.0695\n",
      "Batch [0/18], Loss: 0.8683\n",
      "Epoch [40/100], Loss: 1.0205\n",
      "Batch [0/18], Loss: 1.5265\n",
      "Epoch [41/100], Loss: 0.9853\n",
      "Batch [0/18], Loss: 0.9130\n",
      "Epoch [42/100], Loss: 0.9556\n",
      "Batch [0/18], Loss: 0.8827\n",
      "Epoch [43/100], Loss: 0.9419\n",
      "Batch [0/18], Loss: 0.7623\n",
      "Epoch [44/100], Loss: 0.9300\n",
      "Batch [0/18], Loss: 0.8414\n",
      "Epoch [45/100], Loss: 0.8686\n",
      "Batch [0/18], Loss: 0.7735\n",
      "Epoch [46/100], Loss: 0.8500\n",
      "Batch [0/18], Loss: 0.7999\n",
      "Epoch [47/100], Loss: 0.8585\n",
      "Batch [0/18], Loss: 0.8577\n",
      "Epoch [48/100], Loss: 0.8261\n",
      "Batch [0/18], Loss: 0.6738\n",
      "Epoch [49/100], Loss: 0.7980\n",
      "Batch [0/18], Loss: 0.6631\n",
      "Epoch [50/100], Loss: 0.7835\n",
      "Batch [0/18], Loss: 1.4731\n",
      "Epoch [51/100], Loss: 0.7733\n",
      "Batch [0/18], Loss: 0.7341\n",
      "Epoch [52/100], Loss: 0.7450\n",
      "Batch [0/18], Loss: 0.6080\n",
      "Epoch [53/100], Loss: 0.7097\n",
      "Batch [0/18], Loss: 0.7258\n",
      "Epoch [54/100], Loss: 0.6960\n",
      "Batch [0/18], Loss: 0.7330\n",
      "Epoch [55/100], Loss: 0.6825\n",
      "Batch [0/18], Loss: 0.5580\n",
      "Epoch [56/100], Loss: 0.6640\n",
      "Batch [0/18], Loss: 0.5635\n",
      "Epoch [57/100], Loss: 0.6611\n",
      "Batch [0/18], Loss: 0.5961\n",
      "Epoch [58/100], Loss: 0.6598\n",
      "Batch [0/18], Loss: 0.6469\n",
      "Epoch [59/100], Loss: 0.6244\n",
      "Batch [0/18], Loss: 1.1383\n",
      "Epoch [60/100], Loss: 0.6064\n",
      "Batch [0/18], Loss: 0.5779\n",
      "Epoch [61/100], Loss: 0.5860\n",
      "Batch [0/18], Loss: 0.5370\n",
      "Epoch [62/100], Loss: 0.5893\n",
      "Batch [0/18], Loss: 0.4833\n",
      "Epoch [63/100], Loss: 0.5675\n",
      "Batch [0/18], Loss: 1.3505\n",
      "Epoch [64/100], Loss: 0.5662\n",
      "Batch [0/18], Loss: 0.5030\n",
      "Epoch [65/100], Loss: 0.5373\n",
      "Batch [0/18], Loss: 0.4650\n",
      "Epoch [66/100], Loss: 0.5404\n",
      "Batch [0/18], Loss: 0.5246\n",
      "Epoch [67/100], Loss: 0.5197\n",
      "Batch [0/18], Loss: 0.4825\n",
      "Epoch [68/100], Loss: 0.5040\n",
      "Batch [0/18], Loss: 0.5124\n",
      "Epoch [69/100], Loss: 0.4996\n",
      "Batch [0/18], Loss: 0.4467\n",
      "Epoch [70/100], Loss: 0.4948\n",
      "Batch [0/18], Loss: 0.4868\n",
      "Epoch [71/100], Loss: 0.4823\n",
      "Batch [0/18], Loss: 0.4499\n",
      "Epoch [72/100], Loss: 0.4630\n",
      "Batch [0/18], Loss: 0.4122\n",
      "Epoch [73/100], Loss: 0.4592\n",
      "Batch [0/18], Loss: 0.4113\n",
      "Epoch [74/100], Loss: 0.4564\n",
      "Batch [0/18], Loss: 0.4106\n",
      "Epoch [75/100], Loss: 0.4315\n",
      "Batch [0/18], Loss: 0.3958\n",
      "Epoch [76/100], Loss: 0.4225\n",
      "Batch [0/18], Loss: 0.3328\n",
      "Epoch [77/100], Loss: 0.4510\n",
      "Batch [0/18], Loss: 0.3789\n",
      "Epoch [78/100], Loss: 0.4181\n",
      "Batch [0/18], Loss: 0.3991\n",
      "Epoch [79/100], Loss: 0.4088\n",
      "Batch [0/18], Loss: 0.3489\n",
      "Epoch [80/100], Loss: 0.4144\n",
      "Batch [0/18], Loss: 0.3245\n",
      "Epoch [81/100], Loss: 0.3914\n",
      "Batch [0/18], Loss: 0.8379\n",
      "Epoch [82/100], Loss: 0.3780\n",
      "Batch [0/18], Loss: 0.3176\n",
      "Epoch [83/100], Loss: 0.3699\n",
      "Batch [0/18], Loss: 0.3148\n",
      "Epoch [84/100], Loss: 0.3638\n",
      "Batch [0/18], Loss: 0.3377\n",
      "Epoch [85/100], Loss: 0.3714\n",
      "Batch [0/18], Loss: 0.3501\n",
      "Epoch [86/100], Loss: 0.3499\n",
      "Batch [0/18], Loss: 0.2912\n",
      "Epoch [87/100], Loss: 0.3671\n",
      "Batch [0/18], Loss: 0.3465\n",
      "Epoch [88/100], Loss: 0.3421\n",
      "Batch [0/18], Loss: 0.3242\n",
      "Epoch [89/100], Loss: 0.3368\n",
      "Batch [0/18], Loss: 0.3157\n",
      "Epoch [90/100], Loss: 0.3362\n",
      "Batch [0/18], Loss: 0.3187\n",
      "Epoch [91/100], Loss: 0.3239\n",
      "Batch [0/18], Loss: 0.3247\n",
      "Epoch [92/100], Loss: 0.3262\n",
      "Batch [0/18], Loss: 0.2731\n",
      "Epoch [93/100], Loss: 0.3115\n",
      "Batch [0/18], Loss: 0.3068\n",
      "Epoch [94/100], Loss: 0.3264\n",
      "Batch [0/18], Loss: 0.2842\n",
      "Epoch [95/100], Loss: 0.3103\n",
      "Batch [0/18], Loss: 0.2782\n",
      "Epoch [96/100], Loss: 0.3033\n",
      "Batch [0/18], Loss: 0.2824\n",
      "Epoch [97/100], Loss: 0.2974\n",
      "Batch [0/18], Loss: 0.2892\n",
      "Epoch [98/100], Loss: 0.2888\n",
      "Batch [0/18], Loss: 0.2793\n",
      "Epoch [99/100], Loss: 0.2912\n",
      "Batch [0/18], Loss: 0.2598\n",
      "Epoch [100/100], Loss: 0.2863\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:12:07.731581Z",
     "start_time": "2025-05-28T13:12:06.840337Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 设置生成参数\n",
    "generation_config = {\n",
    "    \"max_length\": 50,  # 生成最大长度（包括输入）\n",
    "    \"eos_token_id\": gpt2_tokenizer.eos_token_id,  # 终止条件\n",
    "    \"pad_token_id\": gpt2_tokenizer.eos_token_id,  # 若需填充，使用EOS的ID\n",
    "    \"do_sample\": True,  # 启用采样\n",
    "    \"temperature\": 0.8,  # 平衡确定性与随机性\n",
    "    \"top_k\": 50,  # 限制候选token数量\n",
    "    \"num_return_sequences\": 2,  # 生成2个不同结果\n",
    "}\n",
    "\n",
    "# 输入提示（可为空或部分诗句）\n",
    "prompt = \"你是谁\"  # 示例：输入半句诗\n",
    "# prompt = \"你能做什么\"  # 示例：输入半句诗\n",
    "# prompt = \"半生长以客为家\"  # 示例：输入半句诗\n",
    "input_ids = gpt2_tokenizer.encode(prompt, return_tensors=\"pt\").to(device)\n",
    "\n",
    "# 生成文本\n",
    "model.to(device)\n",
    "model.eval()\n",
    "outputs = model.generate(\n",
    "    input_ids=input_ids,\n",
    "    **generation_config\n",
    ")\n",
    "\n",
    "# 解码并打印结果\n",
    "for i, output in enumerate(outputs):\n",
    "    poem = gpt2_tokenizer.decode(output, skip_special_tokens=True)\n",
    "    print(f\"生成结果 {i + 1}:\\n{poem}\\n\")"
   ],
   "id": "399b10bb3f9e97c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "生成结果 1:\n",
      "你是谁\n",
      "我是大都督的�答关于周瑜的问题\n",
      "\n",
      "生成结果 2:\n",
      "你是谁\n",
      "我是大都督的�答关于周瑜的问题\n",
      "\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:12:51.936101Z",
     "start_time": "2025-05-28T13:12:51.905125Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 保存LoRA学到的参数\n",
    "model.save_pretrained(\"./gpt2-lora-qa\")"
   ],
   "id": "4cd8e61353edeb5a",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:15:33.990058Z",
     "start_time": "2025-05-28T13:15:33.568822Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from peft import PeftModel\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, GPT2Config, GPT2LMHeadModel\n",
    "from transformers import GPT2Tokenizer\n",
    "import torch\n",
    "\n",
    "gpt2_tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps:0\")\n",
    "\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\"./gpt2_poetry_model\")\n",
    "# 加载LoRA\n",
    "model = PeftModel.from_pretrained(base_model, \"./gpt2-lora-qa\")\n",
    "# model = base_model"
   ],
   "id": "bf43bd09bfc695c7",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-28T13:15:36.643257Z",
     "start_time": "2025-05-28T13:15:35.680982Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 设置生成参数\n",
    "generation_config = {\n",
    "    \"max_length\": 50,  # 生成最大长度（包括输入）\n",
    "    \"eos_token_id\": gpt2_tokenizer.eos_token_id,  # 终止条件\n",
    "    \"pad_token_id\": gpt2_tokenizer.eos_token_id,  # 若需填充，使用EOS的ID\n",
    "    \"do_sample\": True,  # 启用采样\n",
    "    \"temperature\": 0.8,  # 平衡确定性与随机性\n",
    "    \"top_k\": 50,  # 限制候选token数量\n",
    "    \"num_return_sequences\": 2,  # 生成2个不同结果\n",
    "}\n",
    "\n",
    "# 输入提示（可为空或部分诗句）\n",
    "# prompt = \"你是谁\"  # 示例：输入半句诗\n",
    "prompt = \"你能做什么\"  # 示例：输入半句诗\n",
    "# prompt = \"半生长以客为家\"  # 示例：输入半句诗\n",
    "input_ids = gpt2_tokenizer.encode(prompt, return_tensors=\"pt\").to(device)\n",
    "\n",
    "# 生成文本\n",
    "model.to(device)\n",
    "model.eval()\n",
    "outputs = model.generate(\n",
    "    input_ids=input_ids,\n",
    "    **generation_config\n",
    ")\n",
    "\n",
    "# 解码并打印结果\n",
    "for i, output in enumerate(outputs):\n",
    "    poem = gpt2_tokenizer.decode(output, skip_special_tokens=True)\n",
    "    print(f\"生成结果 {i + 1}:\\n{poem}\\n\")"
   ],
   "id": "6e3b016c3c66c0e4",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "生成结果 1:\n",
      "你能做什么\n",
      "我能回答关于周瑜的问题\n",
      "\n",
      "生成结果 2:\n",
      "你能做什么\n",
      "我能回答关于周瑜的问题\n",
      "\n"
     ]
    }
   ],
   "execution_count": 10
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
